package practice.phase03.module01.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;

import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

public class IOHDFS {
    private static FileSystem fs;
    private static Configuration conf;

    @BeforeAll
    static void init() throws URISyntaxException, IOException, InterruptedException {
        conf = new Configuration();
        fs = FileSystem.get(new URI("hdfs://h1:9000"), conf, "root");
    }

    @AfterAll
    static void close() throws IOException {
        fs.close();
    }

    /**
     * 使用IO流上传文件
     *
     * @throws IOException IOException
     */
    @Test
    public void upload() throws IOException {
        // 本地输入流读取文件，HDFS输出流写入文件
        final FileInputStream fileInputStream = new FileInputStream("C:\\Users\\zfqli\\Projects\\big_data_lagou\\lagou_bigdata_assignments\\.gitignore");
        final FSDataOutputStream fsDataOutputStream = fs.create(new Path("/.gitignore"));
        IOUtils.copyBytes(fileInputStream, fsDataOutputStream, 4096, true);
    }

    /**
     * 使用IO流下载文件
     *
     * @throws IOException IOException
     */
    @Test
    public void download() throws IOException {
        // 本地输入流读取文件，HDFS输出流写入文件
        final FileInputStream fileInputStream = new FileInputStream("C:\\Users\\zfqli\\Projects\\big_data_lagou\\lagou_bigdata_assignments\\.gitignore");
        final FSDataOutputStream fsDataOutputStream = fs.create(new Path("/.gitignore"));
        IOUtils.copyBytes(fileInputStream, fsDataOutputStream, 4096, true);
        final FSDataInputStream open = fs.open(new Path("/.gitignore"));
        final FileOutputStream fileOutputStream = new FileOutputStream("C:\\Users\\zfqli\\Projects\\big_data_lagou\\lagou_bigdata_assignments\\.gitignore");
        IOUtils.copyBytes(open, fileOutputStream, conf);
    }

    /**
     * 使用IO流下载文件
     *
     * @throws IOException IOException
     */
    @Test
    public void seekRead() throws IOException {
        // 本地输入流读取文件，HDFS输出流写入文件
        final FSDataInputStream open = fs.open(new Path("/.gitignore"));
        IOUtils.copyBytes(open, System.out, 4096, false);
        open.seek(0); // MR划分文件，分任务处理文件
        IOUtils.copyBytes(open, System.out, 4096, false);
        IOUtils.closeStream(open);
    }
}
